'''
Created on 2012. 11. 18.

@author: love
'''
from gensim import corpora, models, similarities
import logging
import threading
import Queue
import glob

logging.basicConfig(format='%d(asctime)s : %(levelname)s : %s(message)s', level=logging.INFO)

files = glob.glob('./index/*')
file_queue = Queue.Queue()
documents = []
texts = []

class ThreadDocument(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue       
        
    def run(self):          
        while True:              
            try:
                wordlist = []            
                filename = self.queue.get()
                for line in file(filename):
                    for word in line.split():
                        wordlist.append(word)            
                
                documents.append(wordlist)
    
            except Exception as e:
                print e
            
            finally:            
                self.queue.task_done()
                
                
def removeTokensOnce(texts):
    all_tokens = sum(texts, [])
    tokens_once = set(word for word in set(all_tokens) if all_tokens.count(word) == 1)
    texts = [[word for word in text if word not in tokens_once] for text in texts]
    return texts
                       
def main():                        
    for _ in range(20):
        dt = ThreadDocument(file_queue)
        dt.setDaemon(True)
        dt.start()
        
    for fname in files:
        file_queue.put(fname)
    file_queue.join()
    
    texts = removeTokensOnce(documents)
    dictionary = corpora.Dictionary(texts)
    dictionary.save('./trends.dict')
    
    corpus = [dictionary.doc2bow(text) for text in texts]
    corpora.MmCorpus.serialize('./trends.mm', corpus)

if __name__ == '__main__':
    main()